from urllib.robotparser import RobotFileParser
url='https://fgk.chinatax.gov.cn/robots.txt'
rp = RobotFileParser()
rp.set_url(url)
rp.read()
print(rp.can_fetch('Baiduspider', 'https://fgk.chinatax.gov.cn/zcfgk')) 